'use strict'

const { TaiSpider } = require('tai-spider');
const { formatDate } = require('../utils/str-helper');

class NeoesanewsFullSpider extends TaiSpider {

    constructor(options = {}) {
        super(options);
        this.name = 'neoesa-news-full';
        this.debug = true;
        this.start_urls = [
            'https://neo.ssa.esa.int/news-archive'
        ];
    }

    *parse(response) {
        let firstPara = response.css('.pagination').get(0);
        yield* response.follow_all(firstPara.css('a'), this.parse)

        for (let ele of response.css('.news-archives article')) {
            yield response.follow(ele.css('.news-archive__header a'), this.parseArticle);
        }
    }

    *parseArticle(response, spider) {
        let data = response.extract();
        let date = formatDate(data.date)
        data.timestamp = date.valueOf();
        data.date = date.format();

        yield Object.assign({
            index_name: 'tai-article',
        }, data);
    }
}

module.exports = NeoesanewsFullSpider;